<!DOCTYPE html>

<html class="theme-next mist use-motion" lang="zh-Hans">

<head>
  <!-- hexo-inject:begin --><!-- hexo-inject:end --><meta name="generator" content="Hexo 3.9.0">
  <meta charset="UTF-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport"
                                                                                  content="width=device-width, initial-scale=1, maximum-scale=1">
  <meta name="theme-color" content="#222">

  <link rel="stylesheet" href="/lib/needsharebutton/needsharebutton.css">

  <script src="/lib/pace/pace.min.js?v=1.0.2"></script>
  <link href="/lib/pace/pace-theme-minimal.min.css?v=1.0.2" rel="stylesheet">

  <meta http-equiv="Cache-Control" content="no-transform">
  <meta http-equiv="Cache-Control" content="no-siteapp">

  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet"
                                                                                  type="text/css">

  <link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet"
                                                                                  type="text/css">

  <link href="/css/main.css?v=6.0.1" rel="stylesheet" type="text/css">

  <link rel="icon" type="image/png" sizes="16x16"
                                                                                  href="/images/favicon.ico?v=6.0.1">

  <script type="text/javascript" id="hexo.configurations">
    var NexT = window.NexT || {};
    var CONFIG = {
      root: '/',
      scheme: 'Mist',
      version: '6.0.1',
      sidebar: {
        "position": "left",
        "display": "post",
        "offset": 12,
        "b2t": true,
        "scrollpercent": true,
        "onmobile": true
      },
      fancybox: true,
      fastclick: false,
      lazyload: true,
      tabs: true,
      motion: {
        "enable": true,
        "async": false,
        "transition": {
          "post_block": "fadeIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      algolia: {
        applicationID: '',
        apiKey: '',
        indexName: '',
        hits: {
          "per_page": 10
        },
        labels: {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      }
    };

  </script>

  <link rel="stylesheet" href="mermaid.min.css">
  <script src="https://unpkg.com/mermaid@7.1.0/dist/mermaid.min.js"></script>

  <meta name="keywords" content="机器学习,ML,数学,">

  <meta name="description"
                                                                                  content="在第三周的课程里，介绍了Logistic Regression -     逻辑斯谛回归问题，主要应用在Classification -     分类上。还有Regularization -     正则化，如何用来解决Overfitting - 过拟合问题。">
  <meta name="keywords" content="机器学习,ML,数学">
  <meta property="og:type" content="article">
  <meta property="og:title" content="Machine Learning (Week3)">
  <meta property="og:url"
                                                                                  content="https://tankeryang.github.io/posts/Machine Learning (Week3)/index.html">
  <meta property="og:site_name" content="淦">
  <meta property="og:description"
                                                                                  content="在第三周的课程里，介绍了Logistic Regression -     逻辑斯谛回归问题，主要应用在Classification -     分类上。还有Regularization -     正则化，如何用来解决Overfitting - 过拟合问题。">
  <meta property="og:locale" content="zh-Hans">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic0.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic1.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic2.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic3.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic4.jpg">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic2.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic3.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic5.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic6.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic8.jpg">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic7.jpg">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic9.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic10.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic11.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic12.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic13.png">
  <meta property="og:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic14.png">
  <meta property="og:updated_time" content="2019-10-09T17:06:09.571Z">
  <meta name="twitter:card" content="summary">
  <meta name="twitter:title" content="Machine Learning (Week3)">
  <meta name="twitter:description"
                                                                                  content="在第三周的课程里，介绍了Logistic Regression -     逻辑斯谛回归问题，主要应用在Classification -     分类上。还有Regularization -     正则化，如何用来解决Overfitting - 过拟合问题。">
  <meta name="twitter:image"
                                                                                  content="https://tankeryang.github.io/posts/Machine%20Learning%20(Week3)/pic0.png">

  <title>Machine Learning (Week3) | 淦</title>

  <noscript>
    <style type="text/css">
      .use-motion .motion-element,
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-title {
        opacity: initial;
      }

      .use-motion .logo,
      .use-motion .site-title,
      .use-motion .site-subtitle {
        opacity: initial;
        top: initial;
      }

      .use-motion {
        .logo-line-before i {
          left: initial;
        }

        .logo-line-after i {
          right: initial;
        }
      }

    </style>
  </noscript><!-- hexo-inject:begin --><!-- hexo-inject:end -->

</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  <!-- hexo-inject:begin --><!-- hexo-inject:end --><div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope
                                                                                    itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-wrapper">
          <div class="site-meta ">

            <div class="custom-logo-site-title">
              <a href="/" class="brand" rel="start">
                <span class="logo-line-before"><i></i></span>
                <span class="site-title">淦</span>
                <span class="logo-line-after"><i></i></span>
              </a>
            </div>

            <p class="site-subtitle">n*m*lg(b)</p>

          </div>

          <div class="site-nav-toggle">
            <button>
              <span class="btn-bar"></span>
              <span class="btn-bar"></span>
              <span class="btn-bar"></span>
            </button>
          </div>
        </div>

        <nav class="site-nav">

          <ul id="menu" class="menu">

            <li class="menu-item menu-item-home">
              <a href="/" rel="section">
                <i class="menu-item-icon fa fa-fw fa-home"></i> <br>home</a>
            </li>

            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">
                <i class="menu-item-icon fa fa-fw fa-archive"></i>
                <br>archives<span class="badge">16</span>
              </a>
            </li>

            <li class="menu-item menu-item-ml-note">
              <a href="/MachineLearningNote" rel="section">
                <i class="menu-item-icon fa fa-fw fa-book"></i> <br>ML Note</a>
            </li>

            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">
                <i class="menu-item-icon fa fa-fw fa-user"></i> <br>about</a>
            </li>

            <li class="menu-item menu-item-search">

              <a href="javascript:;" class="popup-trigger">

                <i class="menu-item-icon fa fa-search fa-fw"></i> <br>search</a>
            </li>

          </ul>

          <div class="site-search">

            <div class="popup search-popup local-search-popup">
              <div class="local-search-header clearfix">
                <span class="search-icon">
                  <i class="fa fa-search"></i>
                </span>
                <span class="popup-btn-close">
                  <i class="fa fa-times-circle"></i>
                </span>
                <div class="local-search-input-wrapper">
                  <input autocomplete="off" placeholder="searching..." spellcheck="false" type="text"
                                                                                                  id="local-search-input">
                </div>
              </div>
              <div id="local-search-result"></div>
            </div>

          </div>

        </nav>
      </div>
    </header>

    <a href="https://github.com/tankeryang" class="github-corner" target="_blank" title="Follow me on GitHub"
                                                                                    aria-label="Follow me on GitHub"><svg
                                                                                      width="80"
                                                                                      height="80"
                                                                                      viewbox="0 0 250 250"
                                                                                      style="fill:#222; color:#fff; position: absolute; top: 0; border: 0; right: 0;"
                                                                                      aria-hidden="true">
        <path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z" />
        <path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2"
                                                                                        fill="currentColor"
                                                                                        style="transform-origin: 130px 106px;"
                                                                                        class="octo-arm" />
        <path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z"
                                                                                        fill="currentColor"
                                                                                        class="octo-body" />
        </svg></a>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">

            <div id="posts" class="posts-expand">

              <div class="reading-progress-bar"></div>

              <article class="post post-type-normal" itemscope
                                                                                              itemtype="http://schema.org/Article">

                <div class="post-block">
                  <link itemprop="mainEntityOfPage"
                                                                                                  href="https://tankeryang.github.io/posts/Machine Learning (Week3)/">

                  <span hidden itemprop="author" itemscope
                                                                                                  itemtype="http://schema.org/Person">
                    <meta itemprop="name" content="tankeryang">
                    <meta itemprop="description" content>
                    <meta itemprop="image" content="/uploads/girl.jpeg">
                  </span>

                  <span hidden itemprop="publisher" itemscope
                                                                                                  itemtype="http://schema.org/Organization">
                    <meta itemprop="name" content="淦">
                  </span>

                  <header class="post-header">

                    <h1 class="post-title" itemprop="name headline">Machine
                      Learning (Week3)</h1>

                    <div class="post-meta">
                      <span class="post-time">

                        <span class="post-meta-item-icon">
                          <i class="fa fa-calendar-o"></i>
                        </span>

                        <span class="post-meta-item-text">posted on</span>

                        <time title="创建于" itemprop="dateCreated datePublished"
                                                                                                        datetime="2017-10-03T22:26:22+00:00">2017-10-03</time>

                      </span>

                      <span class="post-category">

                        <span class="post-meta-divider">|</span>

                        <span class="post-meta-item-icon">
                          <i class="fa fa-folder-o"></i>
                        </span>

                        <span class="post-meta-item-text">in</span>

                        <span itemprop="about" itemscope
                                                                                                        itemtype="http://schema.org/Thing"><a
                                                                                                          href="/categories/机器学习笔记/"
                                                                                                          itemprop="url"
                                                                                                          rel="index"><span
                                                                                                            itemprop="name">机器学习笔记</span></a></span>

                      </span>

                      <span class="post-comments-count">
                        <span class="post-meta-divider">|</span>
                        <span class="post-meta-item-icon">
                          <i class="fa fa-comment-o"></i>
                        </span>
                        <a href="/posts/Machine Learning (Week3)/#comments"
                                                                                                        itemprop="discussionUrl">
                          <span class="post-comments-count valine-comment-count" data-xid="/posts/Machine Learning (Week3)/"
                                                                                                          itemprop="commentCount"></span>
                        </a>
                      </span>

                      <span id="/posts/Machine Learning (Week3)/" class="leancloud_visitors"
                                                                                                      data-flag-title="Machine Learning (Week3)">
                        <span class="post-meta-divider">|</span>
                        <span class="post-meta-item-icon">
                          <i class="fa fa-eye"></i>
                        </span>

                        <span class="post-meta-item-text">read times&#58;</span>

                        <span class="leancloud-visitors-count"></span>
                      </span>

                      <div class="post-wordcount">

                        <span class="post-meta-item-icon">
                          <i class="fa fa-file-word-o"></i>
                        </span>

                        <span class="post-meta-item-text">words
                          count&#58;</span>

                        <span title="words count">12k</span>

                        <span class="post-meta-divider">|</span>

                        <span class="post-meta-item-icon">
                          <i class="fa fa-clock-o"></i>
                        </span>

                        <span class="post-meta-item-text">minutes to read
                          &asymp;</span>

                        <span title="minutes to read">0:27</span>

                      </div>

                    </div>
                  </header>

                  <div class="post-body" itemprop="articleBody">

                    <p>在第三周的课程里，介绍了<strong>Logistic Regression -
                        逻辑斯谛回归</strong>问题，主要应用在<strong>Classification -
                        分类</strong>上。还有<strong>Regularization -
                        正则化</strong>，如何用来解决<strong>Overfitting - 过拟合</strong>问题。
                    </p>
                    <img src="/posts/Machine%20Learning%20(Week3)/pic0.png">
                    <a id="more"></a>
                    <h1 id="Logistic-Regression-逻辑斯谛回归"><a href="#Logistic-Regression-逻辑斯谛回归" class="headerlink"
                                                                                                      title="Logistic Regression - 逻辑斯谛回归"></a>Logistic
                      Regression - 逻辑斯谛回归</h1>
                    <h2 id="Classification-分类问题"><a href="#Classification-分类问题" class="headerlink"
                                                                                                      title="Classification - 分类问题"></a>Classification
                      - 分类问题</h2>
                    <p>
                      分类在日常中用在很多地方，比如邮件是否垃圾邮件，肿瘤是否良性。通过给定的特征与对应的类别，我们可以训练出一个能够进行分类的算法。相应的，垃圾邮件的特征可以是某些关键词，比如推销类的；而肿瘤的特征可以是肿瘤大小或者别的什么（不懂就不胡说了）。<br>这时我们的结果$y$就是离散化的数字。
                    </p>
                    <ul>
                      <li>如果是<strong>Binary Classification -
                          二分类</strong>问题，$y$可以离散化为$y = 0 or
                        1$，对应<strong>是/否</strong>，<strong>大/小</strong>等抽象的结果。
                      </li>
                      <li>如果是<strong>Multiple Classification -
                          多分类</strong>问题，$y$可以离散化为<strong>元素个数为类别个数的向量</strong>。比如我们需要对某组数据进行分类，训练样本中一共有$n$类，当前训练样本的$y$是属于第$i$类的，则令<span>$y
                          = [0_{1},0_{2},\cdots,1_{i},\cdots,0_{n}]^{T}$</span>
                        <!-- Has MathJax -->，其中下标位置为对应类别。</li>
                    </ul>
                    <p>这里我们先讨论<strong>Binary Classification -
                        二分类</strong>问题。同样先来个例子。</p>
                    <p>假如我们有一组肿瘤大小与其对应性质（良性/恶性）的数据，特征只有一个，就是肿瘤大小，$y=1$和<font
                                                                                                      color="#ff0000">
                        红色X点
                      </font>代表恶性。如下图：<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic1.png"><br>我们看到，假如我们用单纯的线性方程来拟合这组数据，按照图示定义，当<span>$h_{\theta}
                        = \theta^{T}x &gt;0.5$</span>
                      <!-- Has MathJax -->时预测为恶性，则会与原数据<strong>误差较大</strong>。显然单纯的线性方程很难做到精准的分类。
                    </p>
                    <p>
                      我们尝试换一种思路。可以看到，两种类型的数据在某一$x$值上会有明显的区分。在$x$左边是良性的，在$x$右边是恶性的。于是我们做如下分析：<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic2.png"><br>令分界点<span>$x
                        = \frac{-\theta_{0}}{\theta_{1}}$</span>
                      <!-- Has MathJax -->，则当<span>$x &gt;
                        \frac{-\theta_{0}}{\theta_{1}}$</span>
                      <!-- Has MathJax -->即<span>$\theta^{T}x =
                        \theta_{0}+\theta_{1}x &gt;
                        0$</span><!-- Has MathJax -->时，预测$y=1$，当<span>$x &lt;
                        \frac{-\theta_{0}}{\theta_{1}}$</span>
                      <!-- Has MathJax -->即<span>$\theta^{T}x =
                        \theta_{0}+\theta_{1}x &lt;
                        0$</span><!-- Has MathJax -->时，预测$y=0$。这样就能得到很好的分类效果。
                    </p>
                    <p>同样的，多特征时也可以如此这般。比如两个特征时的情况：<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic3.png">
                    </p>
                    <p>因此我们只要一个函数<span>$g(\theta^{T}x)$</span>
                      <!-- Has MathJax -->
                    </p>
                    <ul>
                      <li>当<span>$\theta^{T}x &gt; 0$</span>
                        <!-- Has MathJax -->时，<span>$g(\theta^{T}x) &gt;
                          0.5$</span>
                        <!-- Has MathJax -->
                      </li>
                      <li>当<span>$\theta^{T}x &lt; 0$</span>
                        <!-- Has MathJax -->时，<span>$g(\theta^{T}x) &lt;
                          0.5$</span>
                        <!-- Has MathJax -->
                      </li>
                    </ul>
                    <p>最后令<span>$h_{\theta}(x) = g(\theta^{T}x)$</span>
                      <!-- Has MathJax -->，就ok了。
                    </p>
                    <h2 id="Hypothesis-Representation-假设函数的表示"><a href="#Hypothesis-Representation-假设函数的表示"
                                                                                                      class="headerlink"
                                                                                                      title="Hypothesis Representation - 假设函数的表示"></a>Hypothesis
                      Representation - 假设函数的表示</h2>
                    <p>接着上面的问题。我们给出这样一个函数</p>
                    <center><br><span>$g(z) = \frac{1}{1+e^{-z}}$</span>
                      <!-- Has MathJax --><br>
                    </center>

                    <p>它的图像如下<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic4.jpg">
                    </p>
                    <p>它满足下述性质</p>
                    <ul>
                      <li>当<span>$z &gt; 0$</span>
                        <!-- Has MathJax -->时，<span>$g(z) &gt; 0.5$</span>
                        <!-- Has MathJax -->
                      </li>
                      <li>当<span>$z &lt; 0$</span>
                        <!-- Has MathJax -->时，<span>$g(z) &lt; 0.5$</span>
                        <!-- Has MathJax -->
                      </li>
                    </ul>
                    <p>因此，我们只需令<span>$\theta^{T}x = z$</span>
                      <!-- Has MathJax -->，即</p>
                    <center><br><span>$g(\theta^{T}x) =
                        \frac{1}{1+e^{-\theta^{T}x}}$</span>
                      <!-- Has MathJax --><br></center>

                    <p>则可得我们的假设函数</p>
                    <center><br><span>$h_{\theta}(x) = g(\theta^{T}x) =
                        \frac{1}{1+e^{-\theta^{T}x}}$</span>
                      <!-- Has MathJax --><br></center>

                    <p>这里<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->实际上可以理解为如下</p>
                    <center><br><span>$h_{\theta}(x) = p(y = 1|x;\theta)$</span>
                      <!-- Has MathJax --><br></center>

                    <p>即输入<span>$x$</span>
                      <!-- Has MathJax -->的情况下，预测结果为<span>$1$</span>
                      <!-- Has MathJax -->的概率为<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->。<br>比如说这是一个良性/恶性肿瘤的分类问题，我们训练出了<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->。现在有一个肿瘤的各项特征为<span>$x$</span>
                      <!-- Has MathJax -->，我们要判断它是良性<span>$(y=1)$</span>
                      <!-- Has MathJax -->还是恶性<span>$(y=0)$</span>
                      <!-- Has MathJax -->，把<span>$x$</span>
                      <!-- Has MathJax -->丢进<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->里，结果为<span>$0.8$</span>
                      <!-- Has MathJax -->，那么我们就可以说这个肿瘤有<span>$80\%$</span>
                      <!-- Has MathJax -->的概率是良性的。假如我们设置了一个<strong>阈值</strong>为<span>$0.7$</span>
                      <!-- Has MathJax -->，计算结果超过这个阈值就可以声明预测为真，那么在上面的情况中，我们就可以直接对病人说你的肿瘤是良性的，不用担心。
                    </p>
                    <h2 id="Decision-Boundary-判定边界"><a href="#Decision-Boundary-判定边界" class="headerlink"
                                                                                                      title="Decision Boundary - 判定边界"></a>Decision
                      Boundary - 判定边界</h2>
                    <p>在逻辑斯谛回归中，我们一般</p>
                    <ul>
                      <li>当<span>$h_{\theta}(x) &gt; 0.5$</span>
                        <!-- Has MathJax -->，即当<span>$\theta^{T}x &gt; 0$</span>
                        <!-- Has MathJax -->时，预测<span>$y=1$</span>
                        <!-- Has MathJax -->
                      </li>
                      <li>当<span>$h_{\theta}(x) &lt; 0.5$</span>
                        <!-- Has MathJax -->，即当<span>$\theta^{T}x &lt; 0$</span>
                        <!-- Has MathJax -->时，预测<span>$y=0$</span>
                        <!-- Has MathJax -->
                      </li>
                    </ul>
                    <p>如下图（见上节）<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic2.png"><br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic3.png">
                    </p>
                    <p>此时<strong>阈值</strong>为<span>$0.5$</span>
                      <!-- Has MathJax -->
                    </p>
                    <p>对于一般的问题，<span>$0.5$</span>
                      <!-- Has MathJax -->的阈值足以胜任。可是假如是一些精确度要求很高的问题，就比如刚刚的判断肿瘤是否良性，或者判断是否得癌症等，那么就应该把<strong>阈值</strong>设置得高点，预测结果更准确。毕竟是人命关天的事嘛:)
                    </p>
                    <p>假如我们的样本分布不能线性划分，如下图所示<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic5.png">
                    </p>
                    <p>我们可以用一个非线性的边界（高次多项式）来划分。</p>
                    <h2 id="Cost-function-代价函数"><a href="#Cost-function-代价函数" class="headerlink"
                                                                                                      title="Cost function - 代价函数"></a>Cost
                      function - 代价函数</h2>
                    <p>
                      对于前面的回归问题，我们的代价函数是<strong>所有误差的平方和取均值</strong>（实际上就是<strong>方差</strong>）
                    </p>
                    <center>
                      <br><span>$J(\theta)=\frac{1}{2m}\sum_{i=1}^{m}\left(h_{\theta}(x^{(i)})-y^{(
                        i)}\right)^{2}$</span><!-- Has MathJax --><br></center>

                    <p><br><br>如果我们沿用这个计算方法，将逻辑斯谛回归的<span>$h_{\theta}(x) =
                        g(\theta^{T}x) =
                        \frac{1}{1+e^{-\theta^{T}x}}$</span>
                      <!-- Has MathJax -->代入进上式的话，我们的函数图像会呈现出下面一种状况<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic6.png">
                    </p>
                    <p>
                      这是一个<strong>非凸函数</strong>，它有许多的局部最小值，不利于用梯度下降法寻找全局最小值。<br>所以我们要对逻辑回归重新定义一个代价函数。
                    </p>
                    <p>根据我们<span>$h_{\theta}(x)$</span><!-- Has MathJax -->的性质
                    </p>
                    <ul>
                      <li>当<span>$\theta^{T}x &gt; 0$</span>
                        <!-- Has MathJax -->时，<span>$h_{\theta}(x) &gt;
                          0.5$</span>
                        <!-- Has MathJax -->
                      </li>
                      <li>当<span>$\theta^{T}x &lt; 0$</span>
                        <!-- Has MathJax -->时，<span>$h_{\theta}(x) &lt;
                          0.5$</span>
                        <!-- Has MathJax -->
                      </li>
                      <li><span>$h_{\theta}(x) \in (0, 1)$</span>
                        <!-- Has MathJax -->
                      </li>
                    </ul>
                    <p>我们对代价函数作出如下定义</p>
                    <center><br><span>$$Cost\left( h_{\theta}\left( x \right), y
                        \right)
                        =\begin{cases}
                        -log\left( h_{\theta}\left( x \right ) \right )&amp;
                        \text{ if } y = 1 \\
                        -log\left( 1 - h_{\theta}\left( x \right ) \right )&amp;
                        \text{ if } y = 0
                        \end{cases}$$</span><!-- Has MathJax --><br></center>

                    <p>它的函数图像和意义如下所示</p>
                    <ul>
                      <li>当<span>$y = 1$</span><!-- Has MathJax -->时<img
                                                                                                        src="/posts/Machine%20Learning%20(Week3)/pic8.jpg">
                      </li>
                    </ul>
                    <p>它所反映的就是当样本结果<span>$y = 1$</span>
                      <!-- Has MathJax -->时，如果我们<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->输出也<span>$\rightarrow 1$</span>
                      <!-- Has MathJax -->的话，我们的误差就<span>$\rightarrow 0$</span>
                      <!-- Has MathJax -->；反之，如果我们<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->输出<span>$\rightarrow 0$</span>
                      <!-- Has MathJax -->的话，我们的误差就<span>$\rightarrow
                        \infty$</span>
                      <!-- Has MathJax --><br><br></p>
                    <ul>
                      <li>当<span>$y = 0$</span><!-- Has MathJax -->时<img
                                                                                                        src="/posts/Machine%20Learning%20(Week3)/pic7.jpg">
                      </li>
                    </ul>
                    <p>它所反映的就是当样本结果<span>$y = 0$</span>
                      <!-- Has MathJax -->时，如果我们<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->输出也<span>$\rightarrow 0$</span>
                      <!-- Has MathJax -->的话，我们的误差就<span>$\rightarrow 0$</span>
                      <!-- Has MathJax -->；反之，如果我们<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->输出<span>$\rightarrow 1$</span>
                      <!-- Has MathJax -->的话，我们的误差就<span>$\rightarrow
                        \infty$</span>
                      <!-- Has MathJax -->
                    </p>
                    <p>没毛病:)</p>
                    <h2
                                                                                                    id="Simplified-cost-function-and-gradient-descent-化简代价函数与梯度下降">
                      <a href="#Simplified-cost-function-and-gradient-descent-化简代价函数与梯度下降" class="headerlink"
                                                                                                      title="Simplified cost function and gradient descent - 化简代价函数与梯度下降"></a>Simplified
                      cost function and gradient descent - 化简代价函数与梯度下降</h2>
                    <p>对于<span>$Cost\left( h_{\theta}\left( x \right), y
                        \right)$</span>
                      <!-- Has MathJax -->，我们可以化简成</p>
                    <center><br><span>$Cost\left( h_{\theta}\left( x \right), y
                        \right) =
                        -ylog\left( h_{\theta}\left( x \right) \right) - \left(
                        1-y \right)log\left(
                        1-h_{\theta}\left( x \right) \right)$</span>
                      <!-- Has MathJax --><br>
                    </center>

                    <p><br><br>对于<span>$J(\theta)$</span>
                      <!-- Has MathJax -->我们作如下定义</p>
                    <center><br><span>$J(\theta) = \frac{1}{m}
                        \sum_{i=1}^{m}Cost\left(
                        h_{\theta}\left( x^{\left( i \right)} \right), y^{\left(
                        i \right)
                        }\right)$</span><!-- Has MathJax --><br></center>

                    <p><br><br>将<span>$Cost\left( h_{\theta}\left( x \right), y
                        \right)$</span>
                      <!-- Has MathJax -->代入上式，则可得</p>
                    <center><br><span>$J(\theta) = - \frac{1}{m} \sum_{i=1}^{m}
                        \left[ y^{\left( i
                        \right)}log\left( h_{\theta}\left( x^{\left( i \right)}
                        \right) \right) +
                        \left( 1-y^{\left( i \right)} \right)log\left(
                        1-h_{\theta}\left( x^{\left(
                        i \right)} \right) \right) \right]$</span>
                      <!-- Has MathJax --><br></center>

                    <p>
                      <br><br>这时我们就可以用<strong>梯度下降</strong>来求<span>$min_{\theta}J(\theta)$</span>
                      <!-- Has MathJax -->。</p>
                    <p>与回归一样，我们要做的就是不断更新<span>$\theta$</span>
                      <!-- Has MathJax -->
                    </p>
                    <center><br><span>$\theta := \theta - \alpha \frac{\partial
                        J}{\partial
                        \theta}$</span><!-- Has MathJax --><br></center>

                    <p><br><br>接下来我们来推导一下<span>$\frac{\partial J}{\partial
                        \theta}$</span>
                      <!-- Has MathJax -->。</p>
                    <p>首先，我们有</p>
                    <ul>
                      <li><span>$X_{m \times (n+1)}\begin{bmatrix}x_{0}^{(1)}
                          &amp; x_{1}^{(1)}
                          &amp; \cdots &amp; x_{n}^{(1)}\\ \vdots &amp; \vdots
                          &amp; \ddots &amp;
                          \vdots\\ x_{0}^{(m)} &amp; x_{1}^{(m)} &amp; \cdots
                          &amp;
                          x_{n}^{(m)}\end{bmatrix} = \begin{bmatrix}1 &amp;
                          x_{1}^{(1)} &amp; \cdots
                          &amp; x_{n}^{(1)}\\ \vdots &amp; \vdots &amp; \ddots
                          &amp; \vdots\\ 1
                          &amp; x_{1}^{(m)} &amp; \cdots &amp;
                          x_{n}^{(m)}\end{bmatrix}$</span>
                        <!-- Has MathJax -->
                      </li>
                      <li><span>$Y_{m\times 1} = \begin{bmatrix} y^{(1)} &amp;
                          \cdots &amp;
                          y^{(m)}\end{bmatrix}^{T}$</span><!-- Has MathJax -->
                      </li>
                      <li><span>$\theta = \begin{bmatrix}\theta_{0} &amp;
                          \theta_{1} &amp; \cdots
                          &amp; \theta_{n} \end{bmatrix}^{T}$</span>
                        <!-- Has MathJax -->
                      </li>
                      <li><span>$\frac{\partial J}{\partial \theta} = -
                          \frac{1}{m} \sum_{i=1}^{m}
                          \left[ y^{(i)}\frac{\partial
                          log(h_{\theta}(x^{(i)}))}{\partial \theta} +
                          ( 1-y^{(i)})\frac{\partial log(1-h_{\theta}( x^{(i)}
                          ))}{\partial \theta}
                          \right]$</span><!-- Has MathJax -->
                      </li>
                      <li><span>$h_{\theta}(x^{(i)}) = g(\theta^{T}x^{(i)}) =
                          \frac{1}{1+e^{-\theta^{T}x^{(i)}}}$</span>
                        <!-- Has MathJax -->
                      </li>
                    </ul>
                    <p><br><br>因为</p>
                    <center><br><span>$y^{(i)} \frac{\partial
                        log(h_{\theta}(x^{(i)}))}{\partial
                        \theta} = \frac{y^{(i)}}{h_{\theta}(x^{(i)})} \cdot
                        \frac{x^{(i)}e^{-\theta^{T}x^{(i)}}}{(1+e^{-\theta^{T}x^{(i)}})^{2}}$</span>
                      <!-- Has MathJax --><br><br><br><span>$=
                        y^{(i)}(1+e^{-\theta^{T}x^{(i)}})
                        \cdot
                        \frac{x^{(i)}e^{-\theta^{T}x^{(i)}}}{(1+e^{-\theta^{T}x^{(i)}})^{2}}$</span>
                      <!-- Has MathJax --><br><br><br><span>$= y^{(i)}
                        \frac{x^{(i)}e^{-\theta^{T}x^{(i)}}}{1+e^{-\theta^{T}x^{(i)}}}$</span>
                      <!-- Has MathJax --><br><br><br><span>$= y^{(i)}
                        h_{\theta}(x^{(i)})x^{(i)}e^{-\theta^{T}x^{(i)}}$</span>
                      <!-- Has MathJax --><br></center>

                    <p><br><br>又因为</p>
                    <center><br><span>$(1-y^{(i)}) \frac{\partial
                        log(1-h_{\theta}( x^{(i)}
                        ))}{\partial \theta} = - (1-y^{(i)})
                        \frac{1}{1-h_{\theta}(x^{(i)})} \cdot
                        \frac{x^{(i)}e^{-\theta^{T}x^{(i)}}}{(1+e^{-\theta^{T}x^{(i)}})^{2}}$</span>
                      <!-- Has MathJax --><br><br><br><span>$= - (1-y^{(i)})
                        \frac{1+e^{-\theta^{T}x^{(i)}}}{e^{-\theta^{T}x^{(i)}}}
                        \cdot
                        \frac{x^{(i)}e^{-\theta^{T}x^{(i)}}}{(1+e^{-\theta^{T}x^{(i)}})^{2}}$</span>
                      <!-- Has MathJax --><br><br><br><span>$= - (1-y^{(i)})
                        \frac{x^{(i)}}{1+e^{-\theta^{T}x^{(i)}}}$</span>
                      <!-- Has MathJax --><br><br><br><span>$= -
                        (1-y^{(i)})x^{(i)}h_{\theta}(x^{(i)})$</span>
                      <!-- Has MathJax --><br>
                    </center>

                    <p>
                      <br><br>将上面两式相加，提出<span>$h_{\theta}(x^{(i)})x^{(i)}$</span>
                      <!-- Has MathJax -->，得</p>
                    <center><br><span>$h_{\theta}(x^{(i)})x^{(i)}
                        \left[y^{(i)}(1+e^{-\theta^{T}x^{(i)}}) -
                        1\right]$</span>
                      <!-- Has MathJax --><br><br><br><span>$= x^{(i)}y^{(i)} -
                        x^{(i)}h_{\theta}(x^{(i)})$</span>
                      <!-- Has MathJax --><br><br><br><span>$=
                        x^{(i)} (y^{(i)} - h_{\theta}(x^{(i)}))$</span>
                      <!-- Has MathJax --><br>
                    </center>

                    <p><br><br>将上式代入<span>$\frac{\partial J}{\partial
                        \theta}$</span>
                      <!-- Has MathJax -->，则</p>
                    <center><br><span>$\frac{\partial J}{\partial \theta} =
                        \frac{1}{m}
                        \sum_{i=1}^{m}(h_{\theta}(x^{(i)}) -
                        y^{(i)})x^{(i)}$</span>
                      <!-- Has MathJax --><br></center>

                    <p><br><br>我们惊奇地发现，<span>$\frac{\partial J}{\partial
                        \theta}$</span>
                      <!-- Has MathJax -->的表达式居然跟回归是一样的！这就是数学的魅力！<br>因此，参考<a
                                                                                                      href="https://tankeryang.github.io/posts/Machine%20Learning%20(Week1)/#Gradient-Descent-for-Liner-Regression-线性回归中的梯度下降">week1</a>的推导，可得
                    </p>
                    <center><br><span>$\frac{\partial J}{\partial \theta} =
                        \frac{1}{m}
                        X^{T}(X\theta - Y)$</span>
                      <!-- Has MathJax --><br><br><span>$$\theta :=
                        \theta - \alpha \frac{1}{m} X^{T}(X\theta - Y)$$</span>
                      <!-- Has MathJax --><br></center>

                    <h2 id="Multi-class-classification-One-vs-all-多分类问题：一对多"><a href="#Multi-class-classification-One-vs-all-多分类问题：一对多"
                                                                                                      class="headerlink"
                                                                                                      title="Multi-class classification: One-vs-all - 多分类问题：一对多"></a>Multi-class
                      classification: One-vs-all - 多分类问题：一对多</h2>
                    <p>
                      在实际分类问题中，我们遇到的大多是需要<strong>分多个类</strong>的问题，比如联系人的分类有家人，朋友，同事，同学等等。在可视化图像中，它们可能会呈现出如下的分布<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic9.png">
                    </p>
                    <p>一对多的做法就是我们分别对这三个类训练<span>$h_{\theta}^{(i)}(x)$</span>
                      <!-- Has MathJax -->，其中<span>$i$</span>
                      <!-- Has MathJax -->为类别序号。如下图所示<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic10.png">
                    </p>
                    <p>训练完所有的<span>$h_{\theta}^{(i)}(x)$</span>
                      <!-- Has MathJax -->后，当我们给一组输入特征<span>$x$</span>
                      <!-- Has MathJax -->时，取<strong>最大的</strong><span>$h_{\theta}^{(i)}(x)$</span>
                      <!-- Has MathJax -->作为我们的预测结果。</p>
                    <h1 id="Regularization-正则化"><a href="#Regularization-正则化" class="headerlink"
                                                                                                      title="Regularization - 正则化"></a>Regularization
                      - 正则化</h1>
                    <h2 id="The-problem-of-overfitting-过拟合问题"><a href="#The-problem-of-overfitting-过拟合问题"
                                                                                                      class="headerlink"
                                                                                                      title="The problem of overfitting - 过拟合问题"></a>The
                      problem of overfitting - 过拟合问题</h2>
                    <p>
                      假如我们样本有非常多的特征，我们也许能训练出一个在样本集上表现得很好的假设函数<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->，但是对于新的输入，我们可能不能很好地进行拟合（预测）。这类问题，我们称之为<strong>过拟合</strong>。<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic11.png"><br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic12.png">
                    </p>
                    <p>对于过拟合问题我们一般有下面一些解决方法</p>
                    <ul>
                      <li>减少特征数量</li>
                    </ul>
                    <blockquote>
                      <p>手动剔除一些不必要的特征，或者用一些降维算法（PCA）来自动减少特征数</p>
                    </blockquote>
                    <ul>
                      <li>正则化</li>
                    </ul>
                    <blockquote>
                      <p>保留所有的特征，同时减小参数<span>$\theta$</span>
                        <!-- Has MathJax -->的大小</p>
                    </blockquote>
                    <h2 id="Cost-function-代价函数-1"><a href="#Cost-function-代价函数-1" class="headerlink"
                                                                                                      title="Cost function - 代价函数"></a>Cost
                      function - 代价函数</h2>
                    <p>首先看下面这两种预测函数在样本集上的结果<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic13.png">
                    </p>
                    <p>我们能看到，左边是比较合适的预测函数，而右边则明显过拟合了。</p>
                    <p>这时我们用一个小小的技巧，在我们的误差函数<span>$J(\theta)$</span>
                      <!-- Has MathJax -->后面对<span>$\theta_{3}$</span>
                      <!-- Has MathJax -->和<span>$\theta_{4}$</span>
                      <!-- Has MathJax -->加一个<strong>惩罚系数（或者说补偿反馈）</strong>，使之变为
                    </p>
                    <center>
                      <br><span>$J(\theta)=\frac{1}{2m}\sum_{i=1}^{m}\left(h_{\theta}(x^{(i)})-y^{(i)}\right)^{2}
                        + 1000\theta_{3} + 1000\theta_{4}$</span>
                      <!-- Has MathJax --><br></center>

                    <p><br><br>由上可知，我们要求最优的<span>$\theta$</span>
                      <!-- Has MathJax -->，使得<span>$J(\theta)$</span>
                      <!-- Has MathJax -->取得最小值，那么我们在优化过程中（比如梯度下降）<span>$\theta_{3}$</span>
                      <!-- Has MathJax -->和<span>$\theta_{4}$</span>
                      <!-- Has MathJax -->一定<span>$\rightarrow 0$</span>
                      <!-- Has MathJax -->，因为他们占比很大。这样<span>$\theta_{3}$</span>
                      <!-- Has MathJax -->和<span>$\theta_{4}$</span>
                      <!-- Has MathJax -->对<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->的贡献就非常小，<span>$x^{3}$</span>
                      <!-- Has MathJax -->和<span>$x^{4}$</span>
                      <!-- Has MathJax -->这些高次项在<span>$h_{\theta}(x)$</span>
                      <!-- Has MathJax -->所占的权重就小很多，有效地防止了<strong>过拟合</strong>。
                    </p>
                    <p>假如我们有非常多的特征，不知道要对哪些对应的参数<span>$\theta$</span>
                      <!-- Has MathJax -->作惩罚，那么最好的办法就是对所有的<span>$\theta$</span>
                      <!-- Has MathJax -->作惩罚，然后让程序自己迭代优化。所以我们的代价函数<span>$J(\theta)$</span>
                      <!-- Has MathJax -->就变成下面这种形式</p>
                    <center><br><span>$J(\theta) = \frac{1}{2m} \left[
                        \sum_{i=1}^{m}(h_{\theta}(x^{(i)})-y^{(i)})^{2} +
                        \lambda \sum_{j=1}^{n}
                        \theta_{j}^{2} \right]$</span><!-- Has MathJax --><br>
                    </center>

                    <p><br><br>其中<span>$\lambda$</span>
                      <!-- Has MathJax -->称为<strong>正则化参数</strong>。一般我们不对<span>$\theta_{0}$</span>
                      <!-- Has MathJax -->进行惩罚。</p>
                    <p>正则化后的假设函数如下图所示<br><img
                                                                                                      src="/posts/Machine%20Learning%20(Week3)/pic14.png">
                    </p>
                    <p>其中<font color="#538fca">蓝色</font>曲线是过拟合的情况，<font
                                                                                                      color="#ee34e1">
                        紫色</font>
                      曲线是正则化后的假设函数曲线，而<font color="#ef9a3d">橙色</font>直线则是
                      <strong>正则化参数过大</strong>
                      导致的
                      <strong>欠拟合</strong>。为什么会这样呢？因为正则化参数过大，会对<span>$(\theta_{1}
                        \cdots
                        \theta_{n})$</span>
                      <!-- Has MathJax -->惩罚过重，以至于<span>$(\theta_{1} \cdots
                        \theta_{n}) \rightarrow 0$</span>
                      <!-- Has MathJax -->，使得<span>$h_{\theta}(x)
                        \approx \theta_{0}$</span><!-- Has MathJax -->。</p>
                    <p>因此对于正则化，我们要选一个合适的值，才有好的效果。</p>
                    <h2 id="Regularized-linear-regression-正则化后的线性回归"><a href="#Regularized-linear-regression-正则化后的线性回归"
                                                                                                      class="headerlink"
                                                                                                      title="Regularized linear regression - 正则化后的线性回归"></a>Regularized
                      linear regression - 正则化后的线性回归</h2>
                    <p>正则化后我们的代价函数变成</p>
                    <center><br><span>$J(\theta) = \frac{1}{2m} \left\{
                        \left[\sum_{i=1}^{m}(h_{\theta}(x^{(i)})-y^{(i)})^{2}
                        \right] + \lambda
                        \sum_{j=1}^{n} \theta_{j}^{2} \right\}$</span>
                      <!-- Has MathJax --><br>
                    </center>

                    <p><br><br>如果我们用梯度下降来求最优<span>$\theta$</span>
                      <!-- Has MathJax -->，我们更新<span>$\theta$</span>
                      <!-- Has MathJax -->就要分别更新<span>$\theta_{0}$</span>
                      <!-- Has MathJax -->和<span>$\theta_{1} \cdots
                        \theta_{n}$</span>
                      <!-- Has MathJax -->
                    </p>
                    <center><br><span>$$\begin{cases}
                        \theta_{0} := \theta_{0} - \alpha \frac{1}{m}
                        \sum_{i=1}^{m}(
                        h_{\theta}(x^{(i)})-y^{(i)} )x_{0}^{(i)} \\
                        \theta_{j} := \theta_{j} - \alpha \left\{ \frac{1}{m}
                        \left[\sum_{i=1}^{m}(
                        h_{\theta}(x^{(i)})-y^{(i)} )x_{j}^{(i)}
                        \right]+\frac{\lambda}{m}\theta_{j}\right\} &amp; j=1,2,
                        \cdots ,n
                        \end{cases}$$</span><!-- Has MathJax --><br></center>

                    <p><br><br>其中<span>$\theta_{j}$</span>
                      <!-- Has MathJax -->可以化简成</p>
                    <center><br><span>$\theta_{j}(1-\alpha\frac{\lambda}{m}) -
                        \alpha\frac{1}{m}\sum_{i=1}^{m}(h_{\theta}(x^{(i)})-y^{(i)})x_{j}^{(i)}$</span>
                      <!-- Has MathJax --><br></center>

                    <p><br><br>我们看到，<span>$(1-\alpha\frac{\lambda}{m}) &lt;
                        1$</span>
                      <!-- Has MathJax -->，所以正则化后的梯度下降实际上就是让<span>$\theta_{j}$</span>
                      <!-- Has MathJax -->减少一定的比例后再进行原来的梯度下降。</p>
                    <p>我们知道，梯度<span>$\frac{\partial J}{\partial \theta}$</span>
                      <!-- Has MathJax -->为<span>$0$</span>
                      <!-- Has MathJax -->时，<span>$J(\theta)$</span>
                      <!-- Has MathJax -->取得极小值，所以我们令
                    </p>
                    <center><br><span>$$\begin{cases}
                        \sum_{i=1}^{m}( h_{\theta}(x^{(i)})-y^{(i)} )x_{0}^{(i)}
                        = 0\\
                        \sum_{i=1}^{m}( h_{\theta}(x^{(i)})-y^{(i)} )x_{j}^{(i)}
                        + \lambda\theta_{j}
                        = 0 &amp; j=1,2, \cdots ,n
                        \end{cases}$$</span><!-- Has MathJax --><br></center>

                    <p>即</p>
                    <center><br><span>$\frac{\partial J}{\partial \theta}
                        =\begin{bmatrix}x_{0}^{(1)} &amp; x_{0}^{(2)}
                        &amp;\cdots &amp;
                        x_{0}^{(m)}\\ x_{1}^{(1)} &amp; x_{1}^{(2)} &amp; \cdots
                        &amp; x_{1}^{(m)}\\
                        \vdots &amp; \vdots &amp; \ddots &amp; \vdots \\
                        x_{n}^{(1)} &amp;
                        x_{n}^{(2)} &amp; \cdots &amp; x_{n}^{(m)}\end{bmatrix}
                        \begin{bmatrix}h_{\theta}(x^{(1)})-y^{(1)}\\h_{\theta}(x^{(2)})-y^{(2)}\\
                        \vdots\\h_{\theta}(x^{(m)})-y^{(m)}\end{bmatrix} +
                        \lambda
                        \begin{bmatrix}0\\ \theta_{1}\\ \vdots \\
                        \theta_{n}\end{bmatrix} =
                        0$</span><!-- Has MathJax --><br></center>

                    <p><br><br>也即</p>
                    <center><br><span>$X^{T}(X\theta - Y) + \lambda
                        \begin{bmatrix}0 &amp; &amp;
                        &amp; \\ &amp; 1 &amp; &amp; \\ &amp; &amp; \ddots &amp;
                        \\ &amp; &amp;
                        &amp; 1\end{bmatrix}_{(n+1)^{2}} \theta = 0$</span>
                      <!-- Has MathJax --><br>
                    </center>

                    <p><br><br>去掉括号，并提出<span>$\theta$</span>
                      <!-- Has MathJax -->，整理等式</p>
                    <center><br><span>$(X^{T}X + \lambda\begin{bmatrix}0 &amp;
                        &amp; &amp; \\ &amp;
                        1 &amp; &amp; \\ &amp; &amp; \ddots &amp; \\ &amp; &amp;
                        &amp;
                        1\end{bmatrix}_{(n+1)^{2}}) \theta = X^{T}Y$</span>
                      <!-- Has MathJax --><br>
                    </center>

                    <p><br><br>最后我们可得</p>
                    <center><br><span>$\theta = (X^{T}X +
                        \lambda\begin{bmatrix}0 &amp; &amp; &amp;
                        \\ &amp; 1 &amp; &amp; \\ &amp; &amp; \ddots &amp; \\
                        &amp; &amp; &amp;
                        1\end{bmatrix}_{(n+1)^{2}})^{-1} X^{T}Y$</span>
                      <!-- Has MathJax --><br>
                    </center>

                    <p><br><br>上式就是正则化后的 <strong>Normal equation -
                        正规方程</strong>，其中<span>$(X^{T}X +
                        \lambda\begin{bmatrix}0 &amp; &amp; &amp; \\ &amp; 1
                        &amp; &amp; \\ &amp;
                        &amp; \ddots &amp; \\ &amp; &amp; &amp;
                        1\end{bmatrix}_{(n+1)^{2}})$</span>
                      <!-- Has MathJax -->一定是可逆的。这个就不在此作证明了。</p>
                    <h2 id="Regularized-logistic-regression-正则化后的逻辑斯谛回归"><a href="#Regularized-logistic-regression-正则化后的逻辑斯谛回归"
                                                                                                      class="headerlink"
                                                                                                      title="Regularized logistic regression - 正则化后的逻辑斯谛回归"></a>Regularized
                      logistic regression - 正则化后的逻辑斯谛回归</h2>
                    <p>与线性回归一样，我们在原来的代价函数<span>$J(\theta)$</span>
                      <!-- Has MathJax -->后面加上一个惩罚项，则<span>$J(\theta)$</span>
                      <!-- Has MathJax -->变成
                    </p>
                    <center><br><span>$J(\theta) = - \left\{ \frac{1}{m}
                        \sum_{i=1}^{m} \left[
                        y^{\left( i \right)}log\left( h_{\theta}\left( x^{\left(
                        i \right)} \right)
                        \right) + \left( 1-y^{\left( i \right)} \right)log\left(
                        1-h_{\theta}\left(
                        x^{\left( i \right)} \right) \right) \right] \right\} +
                        \frac{\lambda}{2m}
                        \sum_{j=1}^{2} \theta_{j}^{2}$</span>
                      <!-- Has MathJax --><br></center>

                    <p><br><br>因为其<span>$\frac{\partial J(\theta)}{\partial
                        \theta}$</span>
                      <!-- Has MathJax -->的形式与上面线性回归一样，所以梯度下降的过程同上。</p>
                    <p><br><br>
                      <div class="note info">
                        <p>
                          <center><strong>课程资料</strong></center>
                        </p>
                        <ul>
                          <li><a href="https://github.com/tankeryang/Coursera-machine-learning-lecture-note/tree/master/week3"
                                                                                                            target="_blank"
                                                                                                            rel="noopener">week3课程讲义</a>
                          </li>
                          <li><a href="https://github.com/tankeryang/Coursera-machine-learning-assignment/tree/master/machine-learning-ex2"
                                                                                                            target="_blank"
                                                                                                            rel="noopener">编程作业ex2</a>
                          </li>
                        </ul>
                      </div>
                    </p>

                  </div>

                  <div>

                    <div>

                      <div
                                                                                                      style="text-align:center;color: #ccc;font-size:14px;">
                        ---------------------------------END---------------------------------
                      </div>

                    </div>

                  </div>

                  <div>
                    <div
                                                                                                    style="padding: 10px 0; margin: 20px auto; width: 90%; text-align: center;">
                      <div>手抖一下？</div>
                      <button id="rewardButton" disable="enable"
                                                                                                      onclick="var qr = document.getElementById('QR'); if (qr.style.display === 'none') {qr.style.display='block';} else {qr.style.display='none'}">
                        <span>献爱心</span>
                      </button>
                      <div id="QR" style="display: none;">

                        <div id="wechat" style="display: inline-block">
                          <img id="wechat_qr" src="/images/wx_pay1.png"
                                                                                                          alt="tankeryang WechatPay">
                          <p>WechatPay</p>
                        </div>

                      </div>
                    </div>

                  </div>

                  <footer class="post-footer">

                    <div class="post-tags">

                      <a href="/tags/机器学习/" rel="tag"><i class="fa fa-tag"></i>
                        机器学习</a>

                      <a href="/tags/ML/" rel="tag"><i class="fa fa-tag"></i>
                        ML</a>

                      <a href="/tags/数学/" rel="tag"><i class="fa fa-tag"></i>
                        数学</a>

                    </div>

                    <div class="post-widgets">

                      <div id="needsharebutton-postbottom">
                        <span class="btn">
                          <i class="fa fa-share-alt" aria-hidden="true"></i>
                        </span>
                      </div>

                    </div>

                    <div class="post-nav">
                      <div class="post-nav-next post-nav-item">

                        <a href="/posts/Machine Learning (Week2)/" rel="next"
                                                                                                        title="Machine Learning (Week2)">
                          <i class="fa fa-chevron-left"></i> Machine Learning
                          (Week2)
                        </a>

                      </div>

                      <span class="post-nav-divider"></span>

                      <div class="post-nav-prev post-nav-item">

                        <a href="/posts/半次元爬虫-banciyuan-downloader-v1-0 发布/" rel="prev"
                                                                                                        title="半次元爬虫 banciyuan-downloader v1.0 发布">
                          半次元爬虫 banciyuan-downloader v1.0 发布 <i
                                                                                                          class="fa fa-chevron-right"></i>
                        </a>

                      </div>
                    </div>

                  </footer>
                </div>

              </article>

              <div class="post-spread">

                <!-- Go to www.addthis.com/dashboard to customize your tools -->
                <div class="addthis_inline_share_toolbox">
                  <script type="text/javascript" src="//s7.addthis.com/js/300/addthis_widget.js#pubid=ra-59b14058bcac3b78"
                                                                                                  async="async">
                  </script>
                </div>

              </div>
            </div>

          </div>

          <div class="comments" id="comments">
          </div>

        </div>

        <div class="sidebar-toggle">
          <div class="sidebar-toggle-line-wrap">
            <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
            <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
            <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
          </div>
        </div>

        <aside id="sidebar" class="sidebar">

          <div id="sidebar-dimmer"></div>

          <div class="sidebar-inner">

            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc sidebar-nav-active"
                                                                                              data-target="post-toc-wrap">
                文章目录
              </li>
              <li class="sidebar-nav-overview" data-target="site-overview-wrap">
                站点概览
              </li>
            </ul>

            <section class="site-overview-wrap sidebar-panel">
              <div class="site-overview">
                <div class="site-author motion-element" itemprop="author" itemscope
                                                                                                itemtype="http://schema.org/Person">

                  <a href="/" class="site-author-image" rel="start"
                                                                                                  style="border:none">
                    <img class="site-author-image" itemprop="image" src="/uploads/girl.jpeg"
                                                                                                    alt="tankeryang">
                  </a>

                  <p class="site-author-name" itemprop="name">tankeryang</p>
                  <p class="site-description motion-element"
                                                                                                  itemprop="description">
                    汝亦知射乎？吾射不亦精乎？</p>
                </div>

                <nav class="site-state motion-element">

                  <div class="site-state-item site-state-posts">

                    <a href="/archives/">

                      <span class="site-state-item-count">16</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>

                  <div class="site-state-item site-state-categories">
                    <a href="/categories/index.html">
                      <span class="site-state-item-count">7</span>
                      <span class="site-state-item-name">分类</span>
                    </a>
                  </div>

                  <div class="site-state-item site-state-tags">
                    <a href="/tags/index.html">
                      <span class="site-state-item-count">28</span>
                      <span class="site-state-item-name">标签</span>
                    </a>
                  </div>

                </nav>

                <div class="links-of-author motion-element">

                  <span class="links-of-author-item">
                    <a href="https://github.com/tankeryang" target="_blank"
                                                                                                    title="GitHub">

                      <i class="fa fa-fw fa-github"></i>GitHub</a>
                  </span>

                  <span class="links-of-author-item">
                    <a href="mailto:youngzyang@outlook.com" target="_blank"
                                                                                                    title="E-Mail">

                      <i class="fa fa-fw fa-envelope"></i>E-Mail</a>
                  </span>

                </div>

                <div class="cc-license motion-element" itemprop="license">
                  <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" class="cc-opacity"
                                                                                                  target="_blank">
                    <img src="/images/cc-by-nc-sa.svg" alt="Creative Commons">
                  </a>
                </div>

                <div
                                                                                                class="links-of-blogroll motion-element links-of-blogroll-block">
                  <div class="links-of-blogroll-title">
                    <i class="fa  fa-fw fa-link"></i>
                    博客镜像 & 友情链接
                  </div>
                  <ul class="links-of-blogroll-list">

                    <li class="links-of-blogroll-item">
                      <a href="https://tankeryang.github.io" title="淦 - github"
                                                                                                      target="_blank">淦
                        - github</a>
                    </li>

                    <li class="links-of-blogroll-item">
                      <a href="https://tankeryang.coding.me" title="淦 - coding"
                                                                                                      target="_blank">淦
                        - coding</a>
                    </li>

                    <li class="links-of-blogroll-item">
                      <a href="http://tankeryang.gitee.io" title="淦 - gitee"
                                                                                                      target="_blank">淦
                        - gitee</a>
                    </li>

                    <li class="links-of-blogroll-item">
                      <a href="https://deeeeeeeee.github.io" title="未知"
                                                                                                      target="_blank">未知</a>
                    </li>

                    <li class="links-of-blogroll-item">
                      <a href="https://www.tiexo.cn/" title="简单可依赖"
                                                                                                      target="_blank">简单可依赖</a>
                    </li>

                    <li class="links-of-blogroll-item">
                      <a href="https://darrenliuwei.com" title="刘伟"
                                                                                                      target="_blank">刘伟</a>
                    </li>

                  </ul>
                </div>

              </div>
            </section>

            <!--noindex-->
            <section
                                                                                            class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
              <div class="post-toc">

                <div class="post-toc-content">
                  <ol class="nav">
                    <li class="nav-item nav-level-1"><a class="nav-link"
                                                                                                      href="#Logistic-Regression-逻辑斯谛回归"><span
                                                                                                        class="nav-number">1.</span>
                        <span class="nav-text">Logistic
                          Regression - 逻辑斯谛回归</span></a>
                      <ol class="nav-child">
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Classification-分类问题"><span
                                                                                                            class="nav-number">1.1.</span>
                            <span class="nav-text">Classification
                              - 分类问题</span></a></li>
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Hypothesis-Representation-假设函数的表示"><span
                                                                                                            class="nav-number">1.2.</span>
                            <span class="nav-text">Hypothesis
                              Representation - 假设函数的表示</span></a></li>
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Decision-Boundary-判定边界"><span
                                                                                                            class="nav-number">1.3.</span>
                            <span class="nav-text">Decision
                              Boundary - 判定边界</span></a></li>
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Cost-function-代价函数"><span
                                                                                                            class="nav-number">1.4.</span>
                            <span class="nav-text">Cost
                              function - 代价函数</span></a></li>
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Simplified-cost-function-and-gradient-descent-化简代价函数与梯度下降"><span
                                                                                                            class="nav-number">1.5.</span>
                            <span class="nav-text">Simplified
                              cost function and gradient descent -
                              化简代价函数与梯度下降</span></a></li>
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Multi-class-classification-One-vs-all-多分类问题：一对多"><span
                                                                                                            class="nav-number">1.6.</span>
                            <span class="nav-text">Multi-class
                              classification: One-vs-all - 多分类问题：一对多</span></a>
                        </li>
                      </ol>
                    </li>
                    <li class="nav-item nav-level-1"><a class="nav-link"
                                                                                                      href="#Regularization-正则化"><span
                                                                                                        class="nav-number">2.</span>
                        <span class="nav-text">Regularization
                          - 正则化</span></a>
                      <ol class="nav-child">
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#The-problem-of-overfitting-过拟合问题"><span
                                                                                                            class="nav-number">2.1.</span>
                            <span class="nav-text">The
                              problem of overfitting - 过拟合问题</span></a></li>
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Cost-function-代价函数-1"><span
                                                                                                            class="nav-number">2.2.</span>
                            <span class="nav-text">Cost
                              function - 代价函数</span></a></li>
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Regularized-linear-regression-正则化后的线性回归"><span
                                                                                                            class="nav-number">2.3.</span>
                            <span class="nav-text">Regularized
                              linear regression - 正则化后的线性回归</span></a></li>
                        <li class="nav-item nav-level-2"><a class="nav-link"
                                                                                                          href="#Regularized-logistic-regression-正则化后的逻辑斯谛回归"><span
                                                                                                            class="nav-number">2.4.</span>
                            <span class="nav-text">Regularized
                              logistic regression - 正则化后的逻辑斯谛回归</span></a></li>
                      </ol>
                    </li>
                  </ol>
                </div>

              </div>
            </section>
            <!--/noindex-->

            <div class="back-to-top">
              <i class="fa fa-arrow-up"></i>

              <span id="scrollpercent"><span>0</span>%</span>

            </div>

          </div>
        </aside>

      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <hr>
        <div class="copyright">&copy; 2017 &mdash; <span
                                                                                          itemprop="copyrightYear">2019</span>
          <span class="with-love">
            <i class="fa fa-cog fa-spin fa-1x fa-fw"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">tankeryang</span>

          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-area-chart"></i>
          </span>

          <span title="total count">131k</span>

          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>

          <span title="total time">4:51</span>

        </div>

        <div class="powered-by">Powered by — <a class="theme-link" target="_blank"
                                                                                          href="https://hexo.io">Hexo</a>
        </div>

        <span class="post-meta-divider">|</span>

        <div class="theme-info">theme &mdash; <a class="theme-link" target="_blank"
                                                                                          href="https://github.com/theme-next/hexo-theme-next">NexT.Mist</a>
          v6.0.1</div>

        <div class="busuanzi-count">
          <script async
                                                                                          src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js">
          </script>

          <span class="site-uv">
            <i class="fa fa-user"></i>
            <span class="busuanzi-value" id="busuanzi_value_site_uv"></span>
            visitors
          </span>

        </div>

      </div>
    </footer>

  </div>

  <script type="text/javascript">
    if (Object.prototype.toString.call(window.Promise) !==
      '[object Function]') {
      window.Promise = null;
    }

  </script>

  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  <script type="text/javascript"
                                                                                  src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7">
  </script>

  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1">
  </script>

  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1">
  </script>

  <script type="text/javascript"
                                                                                  src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5">
  </script>

  <script type="text/javascript"
                                                                                  src="/lib/reading_progress/reading_progress.js">
  </script>

  <script type="text/javascript" src="/js/src/utils.js?v=6.0.1"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=6.0.1"></script>

  <script type="text/javascript" src="/js/src/scrollspy.js?v=6.0.1"></script>
  <script type="text/javascript" src="/js/src/post-details.js?v=6.0.1"></script>

  <script type="text/javascript" src="/js/src/bootstrap.js?v=6.0.1"></script>

  <script src="//cdn1.lncld.net/static/js/3.0.4/av-min.js"></script>
  <script src="//unpkg.com/valine/dist/Valine.min.js"></script>

  <script type="text/javascript">
    var GUEST = ['nick', 'mail', 'link'];
    var guest = 'nick,mail,link';
    guest = guest.split(',').filter(item => {
      return GUEST.indexOf(item) > -1;
    });
    new Valine({
      el: '#comments',
      verify: true,
      notify: false,
      appId: 'NiFbgAF3vRyR2BayGJOswb21-gzGzoHsz',
      appKey: 'xGF7qavyOOQGtVACk06SnLrW',
      placeholder: '填上邮箱可以收到回复的邮件提醒哦～',
      avatar: 'retro',
      guest_info: guest,
      pageSize: '10' || 10,
    });

  </script>

  <script type="text/javascript">
    // Popup Window;
    var isfetched = false;
    var isXml = true;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length === 0) {
      search_path = "search.xml";
    } else if (/json$/i.test(search_path)) {
      isXml = false;
    }
    var path = "/" + search_path;
    // monitor main search box;
    var onPopupClose = function(e) {
      $('.popup').hide();
      $('#local-search-input').val('');
      $('.search-result-list').remove();
      $('#no-result').remove();
      $(".local-search-pop-overlay").remove();
      $('body').css('overflow', '');
    }

    function proceedsearch() {
      $("body")
        .append(
          '<div class="search-popup-overlay local-search-pop-overlay"></div>')
        .css('overflow', 'hidden');
      $('.search-popup-overlay').click(onPopupClose);
      $('.popup').toggle();
      var $localSearchInput = $('#local-search-input');
      $localSearchInput.attr("autocapitalize", "none");
      $localSearchInput.attr("autocorrect", "off");
      $localSearchInput.focus();
    }
    // search function;
    var searchFunc = function(path, search_id, content_id) {
      'use strict';
      // start loading animation
      $("body")
        .append(
          '<div class="search-popup-overlay local-search-pop-overlay">' +
          '<div id="search-loading-icon">' +
          '<i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>' +
          '</div>' +
          '</div>')
        .css('overflow', 'hidden');
      $("#search-loading-icon").css('margin', '20% auto 0 auto').css(
        'text-align', 'center');
      // ref: https://github.com/ForbesLindesay/unescape-html
      var unescapeHtml = function(html) {
        return String(html)
          .replace(/&quot;/g, '"')
          .replace(/&#39;/g, '\'')
          .replace(/&#x3A;/g, ':')
          // replace all the other &#x; chars
          .replace(/&#(\d+);/g, function(m, p) {
            return String.fromCharCode(p);
          })
          .replace(/&lt;/g, '<')
          .replace(/&gt;/g, '>')
          .replace(/&amp;/g, '&');
      };
      $.ajax({
        url: path,
        dataType: isXml ? "xml" : "json",
        async: true,
        success: function(res) {
          // get the contents from search data
          isfetched = true;
          $('.popup').detach().appendTo('.header-inner');
          var datas = isXml ? $("entry", res).map(function() {
            return {
              title: $("title", this).text(),
              content: $("content", this).text(),
              url: $("url", this).text()
            };
          }).get() : res;
          var input = document.getElementById(search_id);
          var resultContent = document.getElementById(content_id);
          var inputEventFunction = function() {
            var searchText = input.value.trim().toLowerCase();
            var keywords = searchText.split(/[\s\-]+/);
            if (keywords.length > 1) {
              keywords.push(searchText);
            }
            var resultItems = [];
            if (searchText.length > 0) {
              // perform local searching
              datas.forEach(function(data) {
                var isMatch = false;
                var hitCount = 0;
                var searchTextCount = 0;
                var title = data.title.trim();
                var titleInLowerCase = title.toLowerCase();
                var content = data.content.trim().replace(
                  /<[^>]+>/g, "");
                content = unescapeHtml(content);
                var contentInLowerCase = content.toLowerCase();
                var articleUrl = decodeURIComponent(data.url);
                var indexOfTitle = [];
                var indexOfContent = [];
                // only match articles with not empty titles
                if (title != '') {
                  keywords.forEach(function(keyword) {
                    function getIndexByWord(word, text,
                      caseSensitive) {
                      var wordLen = word.length;
                      if (wordLen === 0) {
                        return [];
                      }
                      var startPosition = 0,
                        position = [],
                        index = [];
                      if (!caseSensitive) {
                        text = text.toLowerCase();
                        word = word.toLowerCase();
                      }
                      while ((position = text.indexOf(word,
                          startPosition)) > -1) {
                        index.push({
                          position: position,
                          word: word
                        });
                        startPosition = position + wordLen;
                      }
                      return index;
                    }
                    indexOfTitle = indexOfTitle.concat(
                      getIndexByWord(keyword,
                        titleInLowerCase, false));
                    indexOfContent = indexOfContent.concat(
                      getIndexByWord(keyword,
                        contentInLowerCase, false));
                  });
                  if (indexOfTitle.length > 0 || indexOfContent
                    .length > 0) {
                    isMatch = true;
                    hitCount = indexOfTitle.length +
                      indexOfContent.length;
                  }
                }
                // show search results
                if (isMatch) {
                  // sort index by position of keyword
                  [indexOfTitle, indexOfContent].forEach(function(
                    index) {
                    index.sort(function(itemLeft, itemRight) {
                      if (itemRight.position !== itemLeft
                        .position) {
                        return itemRight.position -
                          itemLeft.position;
                      } else {
                        return itemLeft.word.length -
                          itemRight.word.length;
                      }
                    });
                  });
                  // merge hits into slices
                  function mergeIntoSlice(text, start, end,
                  index) {
                    var item = index[index.length - 1];
                    var position = item.position;
                    var word = item.word;
                    var hits = [];
                    var searchTextCountInSlice = 0;
                    while (position + word.length <= end && index
                      .length != 0) {
                      if (word === searchText) {
                        searchTextCountInSlice++;
                      }
                      hits.push({
                        position: position,
                        length: word.length
                      });
                      var wordEnd = position + word.length;
                      // move to next position of hit
                      index.pop();
                      while (index.length != 0) {
                        item = index[index.length - 1];
                        position = item.position;
                        word = item.word;
                        if (wordEnd > position) {
                          index.pop();
                        } else {
                          break;
                        }
                      }
                    }
                    searchTextCount += searchTextCountInSlice;
                    return {
                      hits: hits,
                      start: start,
                      end: end,
                      searchTextCount: searchTextCountInSlice
                    };
                  }
                  var slicesOfTitle = [];
                  if (indexOfTitle.length != 0) {
                    slicesOfTitle.push(mergeIntoSlice(title, 0,
                      title.length, indexOfTitle));
                  }
                  var slicesOfContent = [];
                  while (indexOfContent.length != 0) {
                    var item = indexOfContent[indexOfContent
                      .length - 1];
                    var position = item.position;
                    var word = item.word;
                    // cut out 100 characters
                    var start = position - 20;
                    var end = position + 80;
                    if (start < 0) {
                      start = 0;
                    }
                    if (end < position + word.length) {
                      end = position + word.length;
                    }
                    if (end > content.length) {
                      end = content.length;
                    }
                    slicesOfContent.push(mergeIntoSlice(content,
                      start, end, indexOfContent));
                  }
                  // sort slices in content by search text's count and hits' count
                  slicesOfContent.sort(function(sliceLeft,
                    sliceRight) {
                    if (sliceLeft.searchTextCount !==
                      sliceRight.searchTextCount) {
                      return sliceRight.searchTextCount -
                        sliceLeft.searchTextCount;
                    } else if (sliceLeft.hits.length !==
                      sliceRight.hits.length) {
                      return sliceRight.hits.length -
                        sliceLeft.hits.length;
                    } else {
                      return sliceLeft.start - sliceRight
                        .start;
                    }
                  });
                  // select top N slices in content
                  var upperBound = parseInt('-1');
                  if (upperBound >= 0) {
                    slicesOfContent = slicesOfContent.slice(0,
                      upperBound);
                  }
                  // highlight title and content
                  function highlightKeyword(text, slice) {
                    var result = '';
                    var prevEnd = slice.start;
                    slice.hits.forEach(function(hit) {
                      result += text.substring(prevEnd, hit
                        .position);
                      var end = hit.position + hit.length;
                      result += '<b class="search-keyword">' +
                        text.substring(hit.position, end) +
                        '</b>';
                      prevEnd = end;
                    });
                    result += text.substring(prevEnd, slice.end);
                    return result;
                  }
                  var resultItem = '';
                  if (slicesOfTitle.length != 0) {
                    resultItem += "<li><a href='" + articleUrl +
                      "' class='search-result-title'>" +
                      highlightKeyword(title, slicesOfTitle[0]) +
                      "</a>";
                  } else {
                    resultItem += "<li><a href='" + articleUrl +
                      "' class='search-result-title'>" + title +
                      "</a>";
                  }
                  slicesOfContent.forEach(function(slice) {
                    resultItem += "<a href='" + articleUrl +
                      "'>" +
                      "<p class=\"search-result\">" +
                      highlightKeyword(content, slice) +
                      "...</p>" + "</a>";
                  });
                  resultItem += "</li>";
                  resultItems.push({
                    item: resultItem,
                    searchTextCount: searchTextCount,
                    hitCount: hitCount,
                    id: resultItems.length
                  });
                }
              })
            };
            if (keywords.length === 1 && keywords[0] === "") {
              resultContent.innerHTML =
                '<div id="no-result"><i class="fa fa-search fa-5x" /></div>'
            } else if (resultItems.length === 0) {
              resultContent.innerHTML =
                '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>'
            } else {
              resultItems.sort(function(resultLeft, resultRight) {
                if (resultLeft.searchTextCount !== resultRight
                  .searchTextCount) {
                  return resultRight.searchTextCount - resultLeft
                    .searchTextCount;
                } else if (resultLeft.hitCount !== resultRight
                  .hitCount) {
                  return resultRight.hitCount - resultLeft
                    .hitCount;
                } else {
                  return resultRight.id - resultLeft.id;
                }
              });
              var searchResultList =
                '<ul class=\"search-result-list\">';
              resultItems.forEach(function(result) {
                searchResultList += result.item;
              })
              searchResultList += "</ul>";
              resultContent.innerHTML = searchResultList;
            }
          }
          if ('auto' === 'auto') {
            input.addEventListener('input', inputEventFunction);
          } else {
            $('.search-icon').click(inputEventFunction);
            input.addEventListener('keypress', function(event) {
              if (event.keyCode === 13) {
                inputEventFunction();
              }
            });
          }
          // remove loading animation
          $(".local-search-pop-overlay").remove();
          $('body').css('overflow', '');
          proceedsearch();
        }
      });
    }
    // handle and trigger popup window;
    $('.popup-trigger').click(function(e) {
      e.stopPropagation();
      if (isfetched === false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };
    });
    $('.popup-btn-close').click(onPopupClose);
    $('.popup').click(function(e) {
      e.stopPropagation();
    });
    $(document).on('keyup', function(event) {
      var shouldDismissSearchPopup = event.which === 27 &&
        $('.search-popup').is(':visible');
      if (shouldDismissSearchPopup) {
        onPopupClose();
      }
    });

  </script>

  <script src="https://cdn1.lncld.net/static/js/av-core-mini-0.6.4.js"></script>
  <script>
    AV.initialize("NiFbgAF3vRyR2BayGJOswb21-gzGzoHsz",
      "xGF7qavyOOQGtVACk06SnLrW");

  </script>
  <script>
    function showTime(Counter) {
      var query = new AV.Query(Counter);
      var entries = [];
      var $visitors = $(".leancloud_visitors");
      $visitors.each(function() {
        entries.push($(this).attr("id").trim());
      });
      query.containedIn('url', entries);
      query.find()
        .done(function(results) {
          var COUNT_CONTAINER_REF = '.leancloud-visitors-count';
          if (results.length === 0) {
            $visitors.find(COUNT_CONTAINER_REF).text(0);
            return;
          }
          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.get('url');
            var time = item.get('time');
            var element = document.getElementById(url);
            $(element).find(COUNT_CONTAINER_REF).text(time);
          }
          for (var i = 0; i < entries.length; i++) {
            var url = entries[i];
            var element = document.getElementById(url);
            var countSpan = $(element).find(COUNT_CONTAINER_REF);
            if (countSpan.text() == '') {
              countSpan.text(0);
            }
          }
        })
        .fail(function(object, error) {
          console.log("Error: " + error.code + " " + error.message);
        });
    }

    function addCount(Counter) {
      var $visitors = $(".leancloud_visitors");
      var url = $visitors.attr('id').trim();
      var title = $visitors.attr('data-flag-title').trim();
      var query = new AV.Query(Counter);
      query.equalTo("url", url);
      query.find({
        success: function(results) {
          if (results.length > 0) {
            var counter = results[0];
            counter.fetchWhenSave(true);
            counter.increment("time");
            counter.save(null, {
              success: function(counter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(
                  counter.get('time'));
              },
              error: function(counter, error) {
                console.log(
                  'Failed to save Visitor num, with error message: ' +
                  error.message);
              }
            });
          } else {
            var newcounter = new Counter();
            /* Set ACL */
            var acl = new AV.ACL();
            acl.setPublicReadAccess(true);
            acl.setPublicWriteAccess(true);
            newcounter.setACL(acl);
            /* End Set ACL */
            newcounter.set("title", title);
            newcounter.set("url", url);
            newcounter.set("time", 1);
            newcounter.save(null, {
              success: function(newcounter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(
                  newcounter.get('time'));
              },
              error: function(newcounter, error) {
                console.log('Failed to create');
              }
            });
          }
        },
        error: function(error) {
          console.log('Error:' + error.code + " " + error.message);
        }
      });
    }
    $(function() {
      var Counter = AV.Object.extend("Counter");
      if ($('.leancloud_visitors').length == 1) {
        addCount(Counter);
      } else if ($('.post-title-link').length > 1) {
        showTime(Counter);
      }
    });

  </script>

  <script type="text/x-mathjax-config">
    MathJax.Hub.Config({
      tex2jax: {
        inlineMath: [ ['$','$'], ["\\(","\\)"]  ],
        processEscapes: true,
        skipTags: ['script', 'noscript', 'style', 'textarea', 'pre', 'code']
      }
    });
</script>

  <script type="text/x-mathjax-config">
    MathJax.Hub.Queue(function() {
      var all = MathJax.Hub.getAllJax(), i;
        for (i=0; i < all.length; i += 1) {
          all[i].SourceElement().parentNode.className += ' has-jax';
        }
    });
</script>
  <script type="text/javascript"
                                                                                  src="//cdn.bootcss.com/mathjax/2.7.1/latest.js?config=TeX-AMS-MML_HTMLorMML">
  </script>

  <script src="/lib/needsharebutton/needsharebutton.js"></script>

  <script>
    pbOptions = {};
    pbOptions.iconStyle = "box";
    pbOptions.boxForm = "horizontal";
    pbOptions.position = "topCenter";
    pbOptions.networks = "Weibo,Wechat,Douban,QQZone,Twitter,Facebook";
    new needShareButton('#needsharebutton-postbottom', pbOptions);

  </script>

  <script src="/lib/pangu/dist/pangu.min.js?v=3.3"></script>
  <script type="text/javascript">
    pangu.spacingPage();

  </script>

  <script type="text/javascript" src="/js/src/exturl.js?v=6.0.1"></script><!-- hexo-inject:begin --><!-- Begin: Injected MathJax -->
<script type="text/x-mathjax-config">
  MathJax.Hub.Config("");
</script>

<script type="text/x-mathjax-config">
  MathJax.Hub.Queue(function() {
    var all = MathJax.Hub.getAllJax(), i;
    for(i=0; i < all.length; i += 1) {
      all[i].SourceElement().parentNode.className += ' has-jax';
    }
  });
</script>

<script type="text/javascript" src="custom_mathjax_source">
</script>
<!-- End: Injected MathJax -->
<!-- hexo-inject:end -->

</body>

</html>
